import json
import os
import time

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.edge.options import Options
from selenium.webdriver.edge.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# 设置 Edge WebDriver 的路径
edge_driver_path = "C:/xiaoyh/edge_driver/msedgedriver.exe"

# 设置 Edge 浏览器的选项
options = Options()
options.add_argument('--headless')  # 无头模式
options.add_argument('--disable-gpu')  # 禁用GPU加速
options.add_argument('--no-sandbox')  # 解决权限问题

# 启动 Microsoft Edge 浏览器
driver = webdriver.Edge(service=Service(edge_driver_path), options=options)


# 定义写入 JSON 文件的方法
def write_to_json(data, filename):
    """将数据写入 JSON 文件"""
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    with open(filename, "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=4)
    print(f"数据已成功写入到 {filename}")

# 定义动态等待方法
def wait_for_elements(by, value, timeout=10):
    """等待指定元素加载完成"""
    return WebDriverWait(driver, timeout).until(
        EC.presence_of_all_elements_located((by, value))
    )

# 清理非法文件名字符
def sanitize_filename(name):
    """移除文件名中的非法字符"""
    return "".join(c for c in name if c.isalnum() or c in (" ", "_")).strip()

# 定义点击 '查看更多' 按钮的方法
def click_load_more_button(driver, max_clicks=5, wait_time=10):
    try:
        for i in range(max_clicks):  # 最多点击 `max_clicks` 次
            print(f"尝试点击第 {i + 1} 次 '查看更多' 按钮...")
            # 等待按钮可点击
            load_more_button = WebDriverWait(driver, wait_time).until(
                EC.element_to_be_clickable((By.CLASS_NAME, "xpage-more-btn"))
            )
            load_more_button.click()  # 点击按钮
            print("点击成功，等待内容加载...")
            time.sleep(2)  # 等待内容加载
    except Exception as e:
        print(f"'查看更多' 按钮操作异常或已无更多内容: {e}")

try:
    # 打开新华网主页
    url = "http://www.xinhuanet.com/depthobserve/xhqmtt.html"
    driver.get(url)

    # 获取模块导航元素
    nav_elements = wait_for_elements(By.XPATH, "//div[contains(@class,'nav-cont')]//a")
    total_modules = len(nav_elements)
    print(f"检测到 {total_modules} 个模块")
    limit_count = 30  # 限制抓取新闻数量

    for index in range(total_modules):
        try:
            # 每次迭代重新获取模块导航元素
            nav_elements = wait_for_elements(By.XPATH, "//div[contains(@class,'nav-cont')]//a")
            nav = nav_elements[index]

            mod_name = nav.text.strip()
            if not mod_name:
                continue

            # 清理模块名称，确保文件名合法
            sanitized_mod_name = sanitize_filename(mod_name)
            print(f"模块名称: {sanitized_mod_name}")

            mod_url = nav.get_attribute("href")
            print(f"模块链接: {mod_url}")

            # 打开模块链接
            driver.get(mod_url)

            # 获取文章元素
            article_elements = wait_for_elements(By.XPATH, "//div[contains(@class,'list-item')]//div[contains(@class,'tit')]//a", timeout=10)
            data_list = set()  # 存储抓取的数据

            # 如果初始加载的数据不足limit_count条，点击查看更多
            if len(article_elements) < limit_count:
                click_load_more_button(driver,1)

            # 再次获取文章元素，更新数据
            article_elements = wait_for_elements(By.XPATH, "//div[contains(@class,'list-item')]//div[contains(@class,'tit')]//a", timeout=10)
            for article in article_elements[:limit_count]:
                title = article.text.strip()
                link = article.get_attribute("href")
                if title:
                    data_list.add((title, link))

            # 写入数据到 JSON 文件
            if data_list:
                filename = f"output/selenium/xinhuawang/{sanitized_mod_name}.json"
                write_to_json([{"title": title, "link": link} for title, link in data_list], filename)
                print(f"模块 {sanitized_mod_name} 抓取成功，共 {len(data_list)} 条数据")
            else:
                print(f"模块 {sanitized_mod_name} 没有可抓取的文章数据，跳过写入。")

            print(f"{sanitized_mod_name} 模块抓取完成\n{'-' * 60}")
        except Exception as module_error:
            print(f"模块 {index} 抓取失败: {module_error}")
            continue

except Exception as e:
    print("发生错误:", e)
finally:
    driver.quit()
    print("程序已退出")
